package processors;

import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import util.FileUtil;
import util.StringUtil;

public class StopwordsProcessor {
	private ArrayList<String> stopwords;
	
	public StopwordsProcessor(){
		stopwords = FileUtil.loadinFile("STOPWORDS.txt");
	}

	public ArrayList<String> process(ArrayList<String> contents) {
		ArrayList<String> filteredContents = new ArrayList<String>();
		for (int i = 0; i < contents.size(); i++) {
			String tag = contents.get(i);
			ArrayList<String> words = StringUtil.lexicalAnalyze(tag);
			ArrayList<String> wordsAfterStopwords = filterAllStopwords(words);
			String newContent = StringUtil.listToString(wordsAfterStopwords);
			filteredContents.add(newContent);
		}
		return filteredContents;
	}
	private ArrayList<String> filterAllStopwords(ArrayList<String> words) {
		ArrayList<String> filteredWords = new ArrayList<String>();
		for (String word : words) {
			if (!testWord(word))
				continue;
			filteredWords.add(word);

		}
		return filteredWords;
	}

	private boolean testWord(String word) {
		for (String stopword : stopwords) {
			if (word.equalsIgnoreCase(stopword))
				return false;
		}

		if (word.length() > 20)
			return false;
		
		Pattern pattern = Pattern.compile("[\\W]+");
		Matcher matcher = pattern.matcher(word);
		if (matcher.find())
			return false;

		return true;
	}
	
	
}
